{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gzip\n",
    "import subprocess\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse(path):\n",
    "    g = gzip.open(path, 'rb')\n",
    "    for l in g:\n",
    "        yield eval(l)\n",
    "\n",
    "def get_df(path):\n",
    "    i = 0\n",
    "    df = {}\n",
    "    for d in parse(path):\n",
    "        df[i] = d\n",
    "        i += 1\n",
    "    return pd.DataFrame.from_dict(df, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET = 'Office_Products'\n",
    "RAW_PATH = os.path.join('./', DATASET)\n",
    "DATA_FILE = 'reviews_{}_5.json.gz'.format(DATASET)\n",
    "META_FILE = 'meta_{}.json.gz'.format(DATASET)\n",
    "\n",
    "RANDOM_SEED = 0\n",
    "NEG_ITEMS = 99"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data\n",
    "\n",
    "1. Load interaction data and item metadata\n",
    "2. Filter out unuseful items in metadata\n",
    "3. Calculate basic statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading interaction data into ./Office_Products\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 17.6M  100 17.6M    0     0  3329k      0  0:00:05  0:00:05 --:--:-- 4531k\n",
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading item metadata into ./Office_Products\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100 45.3M  100 45.3M    0     0  4951k      0  0:00:09  0:00:09 --:--:-- 7944k\n"
     ]
    }
   ],
   "source": [
    "# download data if not exists\n",
    "\n",
    "if not os.path.exists(RAW_PATH):\n",
    "    subprocess.call('mkdir ' + RAW_PATH, shell=True)\n",
    "if not os.path.exists(os.path.join(RAW_PATH, DATA_FILE)):\n",
    "    print('Downloading interaction data into ' + RAW_PATH)\n",
    "    subprocess.call(\n",
    "        'cd {} && curl -O http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/reviews_{}_5.json.gz'\n",
    "        .format(RAW_PATH, DATASET), shell=True)\n",
    "if not os.path.exists(os.path.join(RAW_PATH, META_FILE)):\n",
    "    print('Downloading item metadata into ' + RAW_PATH)\n",
    "    subprocess.call(\n",
    "        'cd {} && curl -O http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/meta_{}.json.gz'\n",
    "        .format(RAW_PATH, DATASET), shell=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>helpful</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>overall</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>reviewTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A32T2H8150OJLU</td>\n",
       "      <td>B00000JBLH</td>\n",
       "      <td>ARH</td>\n",
       "      <td>[3, 4]</td>\n",
       "      <td>I bought my first HP12C in about 1984 or so, a...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>A solid performer, and long time friend</td>\n",
       "      <td>1094169600</td>\n",
       "      <td>09 3, 2004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A3MAFS04ZABRGO</td>\n",
       "      <td>B00000JBLH</td>\n",
       "      <td>Let it Be \"Alan\"</td>\n",
       "      <td>[7, 9]</td>\n",
       "      <td>WHY THIS BELATED REVIEW? I feel very obliged t...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Price of GOLD is up, so don't bury the golden ...</td>\n",
       "      <td>1197676800</td>\n",
       "      <td>12 15, 2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1F1A0QQP2XVH5</td>\n",
       "      <td>B00000JBLH</td>\n",
       "      <td>Mark B</td>\n",
       "      <td>[3, 3]</td>\n",
       "      <td>I have an HP 48GX that has been kicking for mo...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Good functionality, but not durable like old HPs</td>\n",
       "      <td>1293840000</td>\n",
       "      <td>01 1, 2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A49R5DBXXQDE5</td>\n",
       "      <td>B00000JBLH</td>\n",
       "      <td>R. D Johnson</td>\n",
       "      <td>[7, 8]</td>\n",
       "      <td>I've started doing more finance stuff recently...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>One of the last of an almost extinct species</td>\n",
       "      <td>1145404800</td>\n",
       "      <td>04 19, 2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A2XRMQA6PJ5ZJ8</td>\n",
       "      <td>B00000JBLH</td>\n",
       "      <td>Roger J. Buffington</td>\n",
       "      <td>[0, 0]</td>\n",
       "      <td>For simple calculations and discounted cash fl...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Still the best</td>\n",
       "      <td>1375574400</td>\n",
       "      <td>08 4, 2013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       reviewerID        asin         reviewerName helpful  \\\n",
       "0  A32T2H8150OJLU  B00000JBLH                  ARH  [3, 4]   \n",
       "1  A3MAFS04ZABRGO  B00000JBLH     Let it Be \"Alan\"  [7, 9]   \n",
       "2  A1F1A0QQP2XVH5  B00000JBLH               Mark B  [3, 3]   \n",
       "3   A49R5DBXXQDE5  B00000JBLH         R. D Johnson  [7, 8]   \n",
       "4  A2XRMQA6PJ5ZJ8  B00000JBLH  Roger J. Buffington  [0, 0]   \n",
       "\n",
       "                                          reviewText  overall  \\\n",
       "0  I bought my first HP12C in about 1984 or so, a...      5.0   \n",
       "1  WHY THIS BELATED REVIEW? I feel very obliged t...      5.0   \n",
       "2  I have an HP 48GX that has been kicking for mo...      2.0   \n",
       "3  I've started doing more finance stuff recently...      5.0   \n",
       "4  For simple calculations and discounted cash fl...      5.0   \n",
       "\n",
       "                                             summary  unixReviewTime  \\\n",
       "0            A solid performer, and long time friend      1094169600   \n",
       "1  Price of GOLD is up, so don't bury the golden ...      1197676800   \n",
       "2   Good functionality, but not durable like old HPs      1293840000   \n",
       "3       One of the last of an almost extinct species      1145404800   \n",
       "4                                     Still the best      1375574400   \n",
       "\n",
       "    reviewTime  \n",
       "0   09 3, 2004  \n",
       "1  12 15, 2007  \n",
       "2   01 1, 2011  \n",
       "3  04 19, 2006  \n",
       "4   08 4, 2013  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df = get_df(os.path.join(RAW_PATH, DATA_FILE))\n",
    "data_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>imUrl</th>\n",
       "      <th>related</th>\n",
       "      <th>salesRank</th>\n",
       "      <th>categories</th>\n",
       "      <th>title</th>\n",
       "      <th>brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0078800242</td>\n",
       "      <td>All in one TeacherWorks Plus CD-ROM</td>\n",
       "      <td>93.06</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41K1aBkl...</td>\n",
       "      <td>{'buy_after_viewing': ['007861970X']}</td>\n",
       "      <td>{'Software': 18529}</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, C...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0113000316</td>\n",
       "      <td>High quality inkjet cartridges use high-densit...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51AMwP3D...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, P...</td>\n",
       "      <td>123GetInk -14-pack 5-black 3-cyan 3-magenta 3-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>043928631X</td>\n",
       "      <td>Harry Potter living bookmark showing Harry, He...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41SulB7T...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, L...</td>\n",
       "      <td>Harry Potter Lenticular Hologram Bookmark - Ha...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0439340039</td>\n",
       "      <td>Windows based computer game.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51zQE0w%...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'Software': 32784}</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, E...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0439394058</td>\n",
       "      <td>126 pieces: 23\" tall schoolhouse calendar, 12 ...</td>\n",
       "      <td>11.64</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51DFp0Lg...</td>\n",
       "      <td>{'also_bought': ['B000QE1HHU', 'B00207MG4Y', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, E...</td>\n",
       "      <td>Scholastic SC939405 All-In-One Schoolhouse Cal...</td>\n",
       "      <td>Scholastic</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         asin                                        description  price  \\\n",
       "0  0078800242                All in one TeacherWorks Plus CD-ROM  93.06   \n",
       "1  0113000316  High quality inkjet cartridges use high-densit...    NaN   \n",
       "2  043928631X  Harry Potter living bookmark showing Harry, He...    NaN   \n",
       "3  0439340039                       Windows based computer game.    NaN   \n",
       "4  0439394058  126 pieces: 23\" tall schoolhouse calendar, 12 ...  11.64   \n",
       "\n",
       "                                               imUrl  \\\n",
       "0  http://ecx.images-amazon.com/images/I/41K1aBkl...   \n",
       "1  http://ecx.images-amazon.com/images/I/51AMwP3D...   \n",
       "2  http://ecx.images-amazon.com/images/I/41SulB7T...   \n",
       "3  http://ecx.images-amazon.com/images/I/51zQE0w%...   \n",
       "4  http://ecx.images-amazon.com/images/I/51DFp0Lg...   \n",
       "\n",
       "                                             related            salesRank  \\\n",
       "0              {'buy_after_viewing': ['007861970X']}  {'Software': 18529}   \n",
       "1                                                NaN                  NaN   \n",
       "2                                                NaN                  NaN   \n",
       "3                                                NaN  {'Software': 32784}   \n",
       "4  {'also_bought': ['B000QE1HHU', 'B00207MG4Y', '...                  NaN   \n",
       "\n",
       "                                          categories  \\\n",
       "0  [[Office Products, Office & School Supplies, C...   \n",
       "1  [[Office Products, Office & School Supplies, P...   \n",
       "2  [[Office Products, Office & School Supplies, L...   \n",
       "3  [[Office Products, Office & School Supplies, E...   \n",
       "4  [[Office Products, Office & School Supplies, E...   \n",
       "\n",
       "                                               title       brand  \n",
       "0                                                NaN         NaN  \n",
       "1  123GetInk -14-pack 5-black 3-cyan 3-magenta 3-...         NaN  \n",
       "2  Harry Potter Lenticular Hologram Bookmark - Ha...         NaN  \n",
       "3                                                NaN         NaN  \n",
       "4  Scholastic SC939405 All-In-One Schoolhouse Cal...  Scholastic  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_df = get_df(os.path.join(RAW_PATH, META_FILE))\n",
    "meta_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only retain items that appear in interaction data\n",
    "\n",
    "useful_meta_df = meta_df[meta_df['asin'].isin(data_df['asin'])].reset_index(drop=True)\n",
    "all_items = set(useful_meta_df['asin'].values.tolist())\n",
    "\n",
    "def related_filter(related_dict):\n",
    "    out_dict = dict()\n",
    "    if related_dict is not np.nan:\n",
    "        for r in related_dict:\n",
    "            out_dict[r] = list(all_items & set(related_dict[r]))\n",
    "    return out_dict\n",
    "\n",
    "useful_meta_df['related'] = useful_meta_df['related'].apply(related_filter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_users = data_df['reviewerID'].value_counts().size\n",
    "n_items = data_df['asin'].value_counts().size\n",
    "n_clicks = len(data_df)\n",
    "min_time = data_df['unixReviewTime'].min()\n",
    "max_time = data_df['unixReviewTime'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Users: 4905\n",
      "# Items: 2420\n",
      "# Interactions: 53258\n",
      "Time Span: 2000-09-29/2014-07-23\n"
     ]
    }
   ],
   "source": [
    "time_format = '%Y-%m-%d'\n",
    "\n",
    "print('# Users:', n_users)\n",
    "print('# Items:', n_items)\n",
    "print('# Interactions:', n_clicks)\n",
    "print('Time Span: {}/{}'.format(\n",
    "    datetime.utcfromtimestamp(min_time).strftime(time_format),\n",
    "    datetime.utcfromtimestamp(max_time).strftime(time_format))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Dataset\n",
    "\n",
    "### Interaction data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A12DQZKRKTNF5E</td>\n",
       "      <td>B0000AG93P</td>\n",
       "      <td>970185600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A3TB9HNQR54B5V</td>\n",
       "      <td>B00004VVIX</td>\n",
       "      <td>983404800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A2DN7RUNX06BZ1</td>\n",
       "      <td>B00004VVIX</td>\n",
       "      <td>996105600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A37MH7ICH80QOX</td>\n",
       "      <td>B000059RDW</td>\n",
       "      <td>996278400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A3S87ZOPB3UM9N</td>\n",
       "      <td>B00004Z6NA</td>\n",
       "      <td>1007078400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          user_id     item_id        time\n",
       "0  A12DQZKRKTNF5E  B0000AG93P   970185600\n",
       "1  A3TB9HNQR54B5V  B00004VVIX   983404800\n",
       "2  A2DN7RUNX06BZ1  B00004VVIX   996105600\n",
       "3  A37MH7ICH80QOX  B000059RDW   996278400\n",
       "4  A3S87ZOPB3UM9N  B00004Z6NA  1007078400"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_df = data_df.rename(columns={'asin': 'item_id', 'reviewerID': 'user_id', 'unixReviewTime': 'time'})\n",
    "out_df = out_df[['user_id', 'item_id', 'time']]\n",
    "out_df = out_df.drop_duplicates(['user_id', 'item_id', 'time'])\n",
    "out_df = out_df.sort_values(by=['time', 'user_id'], kind='mergesort').reset_index(drop=True)\n",
    "out_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>970185600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>983404800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>996105600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>996278400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1007078400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id        time\n",
       "0        1        1   970185600\n",
       "1        2        2   983404800\n",
       "2        3        2   996105600\n",
       "3        4        3   996278400\n",
       "4        5        4  1007078400"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reindex (start from 1)\n",
    "\n",
    "# uids = sorted(out_df['user_id'].unique())\n",
    "uids = out_df['user_id'].unique()\n",
    "user2id = dict(zip(uids, range(1, len(uids) + 1)))\n",
    "# iids = sorted(out_df['item_id'].unique())\n",
    "iids = out_df['item_id'].unique()\n",
    "item2id = dict(zip(iids, range(1, len(iids) + 1)))\n",
    "\n",
    "out_df['user_id'] = out_df['user_id'].apply(lambda x: user2id[x])\n",
    "out_df['item_id'] = out_df['item_id'].apply(lambda x: item2id[x])\n",
    "out_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# leave one out spliting\n",
    "\n",
    "clicked_item_set = dict()\n",
    "for user_id, seq_df in out_df.groupby('user_id'):\n",
    "    clicked_item_set[user_id] = set(seq_df['item_id'].values.tolist())\n",
    "    \n",
    "def generate_dev_test(data_df):\n",
    "    result_dfs = []\n",
    "    n_items = data_df['item_id'].value_counts().size\n",
    "    for idx in range(2):\n",
    "        result_df = data_df.groupby('user_id').tail(1).copy()\n",
    "        data_df = data_df.drop(result_df.index)\n",
    "        neg_items = np.random.randint(1, n_items + 1, (len(result_df), NEG_ITEMS))\n",
    "        for i, uid in enumerate(result_df['user_id'].values):\n",
    "            user_clicked = clicked_item_set[uid]\n",
    "            for j in range(len(neg_items[i])):\n",
    "                while neg_items[i][j] in user_clicked:\n",
    "                    neg_items[i][j] = np.random.randint(1, n_items + 1)\n",
    "        result_df['neg_items'] = neg_items.tolist()\n",
    "        result_dfs.append(result_df)\n",
    "    return result_dfs, data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(43448, 4905, 4905)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "leave_df = out_df.groupby('user_id').head(1)\n",
    "data_df = out_df.drop(leave_df.index)\n",
    "\n",
    "[test_df, dev_df], data_df = generate_dev_test(data_df)\n",
    "train_df = pd.concat([leave_df, data_df]).sort_index()\n",
    "\n",
    "len(train_df), len(dev_df), len(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>970185600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>983404800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>996105600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>996278400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1007078400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id        time\n",
       "0        1        1   970185600\n",
       "1        2        2   983404800\n",
       "2        3        2   996105600\n",
       "3        4        3   996278400\n",
       "4        5        4  1007078400"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>time</th>\n",
       "      <th>neg_items</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>21</td>\n",
       "      <td>27</td>\n",
       "      <td>1070582400</td>\n",
       "      <td>[1654, 836, 764, 1732, 1034, 278, 1779, 1829, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322</th>\n",
       "      <td>203</td>\n",
       "      <td>120</td>\n",
       "      <td>1170374400</td>\n",
       "      <td>[87, 873, 2060, 308, 1105, 2081, 1207, 1153, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>334</td>\n",
       "      <td>189</td>\n",
       "      <td>1191024000</td>\n",
       "      <td>[866, 1955, 1113, 2082, 2281, 768, 2085, 1793,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>550</th>\n",
       "      <td>71</td>\n",
       "      <td>180</td>\n",
       "      <td>1197763200</td>\n",
       "      <td>[874, 787, 1315, 438, 1848, 1533, 1021, 942, 4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1384</th>\n",
       "      <td>190</td>\n",
       "      <td>402</td>\n",
       "      <td>1240099200</td>\n",
       "      <td>[1732, 2262, 2151, 2376, 596, 1248, 2305, 1132...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id  item_id        time  \\\n",
       "35         21       27  1070582400   \n",
       "322       203      120  1170374400   \n",
       "484       334      189  1191024000   \n",
       "550        71      180  1197763200   \n",
       "1384      190      402  1240099200   \n",
       "\n",
       "                                              neg_items  \n",
       "35    [1654, 836, 764, 1732, 1034, 278, 1779, 1829, ...  \n",
       "322   [87, 873, 2060, 308, 1105, 2081, 1207, 1153, 2...  \n",
       "484   [866, 1955, 1113, 2082, 2281, 768, 2085, 1793,...  \n",
       "550   [874, 787, 1315, 438, 1848, 1533, 1021, 942, 4...  \n",
       "1384  [1732, 2262, 2151, 2376, 596, 1248, 2305, 1132...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save results\n",
    "\n",
    "train_df.to_csv(os.path.join(RAW_PATH, 'train.csv'), sep='\\t', index=False)\n",
    "dev_df.to_csv(os.path.join(RAW_PATH, 'dev.csv'), sep='\\t', index=False)\n",
    "test_df.to_csv(os.path.join(RAW_PATH, 'test.csv'), sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Item Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# level-2 category\n",
    "\n",
    "l2_cate_lst = list()\n",
    "for cate_lst in useful_meta_df['categories']:\n",
    "    l2_cate_lst.append(cate_lst[0][2] if len(cate_lst[0]) > 2 else np.nan)\n",
    "useful_meta_df['l2_category'] = l2_cate_lst  \n",
    "l2_cates = sorted(useful_meta_df['l2_category'].dropna().unique())\n",
    "l2_dict = dict(zip(l2_cates, range(1, len(l2_cates) + 1)))\n",
    "useful_meta_df['l2_category'] = useful_meta_df['l2_category'].apply(lambda x: l2_dict[x] if x == x else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# brand\n",
    "\n",
    "brand_lst = list()\n",
    "for brand in useful_meta_df['brand']:\n",
    "    brand_lst.append(brand if not pd.isnull(brand) else np.nan)\n",
    "useful_meta_df['l1_brand'] = brand_lst\n",
    "# brands = sorted(useful_meta_df['brand'].dropna().unique())\n",
    "brands = useful_meta_df['l1_brand'].dropna().unique()\n",
    "brand_dict = dict(zip(brands, range(1, len(brands) + 1)))\n",
    "useful_meta_df['l1_brand'] = useful_meta_df['l1_brand'].apply(lambda x: brand_dict[x] if x == x else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>imUrl</th>\n",
       "      <th>related</th>\n",
       "      <th>salesRank</th>\n",
       "      <th>categories</th>\n",
       "      <th>title</th>\n",
       "      <th>brand</th>\n",
       "      <th>l2_category</th>\n",
       "      <th>l1_brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B00000JBLU</td>\n",
       "      <td>The classic mini-desktop calculator for home, ...</td>\n",
       "      <td>10.05</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51cpdDRB...</td>\n",
       "      <td>{'also_bought': ['B000MFJNVK', 'B000Y52D5G', '...</td>\n",
       "      <td>{}</td>\n",
       "      <td>[[Office Products, Office Electronics, Calcula...</td>\n",
       "      <td>Texas Instruments TI-1795 SV Standard Function...</td>\n",
       "      <td>Texas Instruments</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B00000JBNX</td>\n",
       "      <td>The new two-line TI-30X IIS with the memory re...</td>\n",
       "      <td>8.38</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/411XXAyD...</td>\n",
       "      <td>{'also_bought': ['B003155XYO', 'B004F9QBGE', '...</td>\n",
       "      <td>{}</td>\n",
       "      <td>[[Office Products, Office Electronics, Calcula...</td>\n",
       "      <td>Texas Instruments TI-30X IIS 2-Line Scientific...</td>\n",
       "      <td>Texas Instruments</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B00000JBO8</td>\n",
       "      <td>Perhaps if we'd had this calculator in high sc...</td>\n",
       "      <td>104.99</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51QZKWB0...</td>\n",
       "      <td>{'also_bought': [], 'bought_together': [], 'bu...</td>\n",
       "      <td>{}</td>\n",
       "      <td>[[Office Products, Office Electronics, Calcula...</td>\n",
       "      <td>Texas Instruments TI-83 Graphing Calculator</td>\n",
       "      <td>Texas Instruments</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B00000JBLH</td>\n",
       "      <td>If you bought yourself a financial calculator ...</td>\n",
       "      <td>51.94</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/415w5D26...</td>\n",
       "      <td>{'also_bought': ['B00000JZKB', 'B001PLII3E', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office Electronics, Calcula...</td>\n",
       "      <td>HP 12C Financial Calculator</td>\n",
       "      <td>HP</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B00000JFNV</td>\n",
       "      <td>Perfect for invitations, thank you notes, movi...</td>\n",
       "      <td>8.42</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51q11PWY...</td>\n",
       "      <td>{'also_bought': ['B00004Z5QO', 'B0000721Z3', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, P...</td>\n",
       "      <td>Avery Half-Fold Greeting Cards for Inkjet Prin...</td>\n",
       "      <td>Avery</td>\n",
       "      <td>25</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2415</th>\n",
       "      <td>B00K5UZP2S</td>\n",
       "      <td></td>\n",
       "      <td>529.99</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41KZwQuZ...</td>\n",
       "      <td>{'also_bought': ['B000SDY14K', 'B00I4XMEYA'], ...</td>\n",
       "      <td>{}</td>\n",
       "      <td>[[Office Products, Office Electronics, Printer...</td>\n",
       "      <td>Brother Printer MFC-L8850CDW Wireless Color La...</td>\n",
       "      <td>Brother</td>\n",
       "      <td>29</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2416</th>\n",
       "      <td>B00KLLBEPI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.99</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41a7pEjw...</td>\n",
       "      <td>{'also_viewed': ['B004F9QBGE', 'B00DGY5IP4', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, P...</td>\n",
       "      <td>Classic Glow Filler Paper, Wide Ruled,600, MAD...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2417</th>\n",
       "      <td>B00KOO594O</td>\n",
       "      <td></td>\n",
       "      <td>54.42</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41wjtN1j...</td>\n",
       "      <td>{'also_viewed': ['B004QWZINW'], 'buy_after_vie...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, F...</td>\n",
       "      <td>SentrySafe CHW20221 Medium Chest Safe, Charcoa...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2418</th>\n",
       "      <td>B00KOO599O</td>\n",
       "      <td></td>\n",
       "      <td>79.43</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51IqUfxT...</td>\n",
       "      <td>{'also_viewed': ['B000Z8W7I0', 'B002ONB4GA', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, F...</td>\n",
       "      <td>SentrySafe FHW40220 Large File Safe, Charcoal ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2419</th>\n",
       "      <td>B00KYA0RC2</td>\n",
       "      <td>Accuteck ShipPro W-8580 110lbs x 0.1 oz. Black...</td>\n",
       "      <td>25.99</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51nM60Wv...</td>\n",
       "      <td>{'also_viewed': ['B003MZORR2', 'B00I9D5IFM', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, E...</td>\n",
       "      <td>Accuteck ShipPro 110lbs x 0.1 oz. Digital Ship...</td>\n",
       "      <td>Accuteck ShipPro</td>\n",
       "      <td>16</td>\n",
       "      <td>341</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2420 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            asin                                        description   price  \\\n",
       "0     B00000JBLU  The classic mini-desktop calculator for home, ...   10.05   \n",
       "1     B00000JBNX  The new two-line TI-30X IIS with the memory re...    8.38   \n",
       "2     B00000JBO8  Perhaps if we'd had this calculator in high sc...  104.99   \n",
       "3     B00000JBLH  If you bought yourself a financial calculator ...   51.94   \n",
       "4     B00000JFNV  Perfect for invitations, thank you notes, movi...    8.42   \n",
       "...          ...                                                ...     ...   \n",
       "2415  B00K5UZP2S                                                     529.99   \n",
       "2416  B00KLLBEPI                                                NaN   10.99   \n",
       "2417  B00KOO594O                                                      54.42   \n",
       "2418  B00KOO599O                                                      79.43   \n",
       "2419  B00KYA0RC2  Accuteck ShipPro W-8580 110lbs x 0.1 oz. Black...   25.99   \n",
       "\n",
       "                                                  imUrl  \\\n",
       "0     http://ecx.images-amazon.com/images/I/51cpdDRB...   \n",
       "1     http://ecx.images-amazon.com/images/I/411XXAyD...   \n",
       "2     http://ecx.images-amazon.com/images/I/51QZKWB0...   \n",
       "3     http://ecx.images-amazon.com/images/I/415w5D26...   \n",
       "4     http://ecx.images-amazon.com/images/I/51q11PWY...   \n",
       "...                                                 ...   \n",
       "2415  http://ecx.images-amazon.com/images/I/41KZwQuZ...   \n",
       "2416  http://ecx.images-amazon.com/images/I/41a7pEjw...   \n",
       "2417  http://ecx.images-amazon.com/images/I/41wjtN1j...   \n",
       "2418  http://ecx.images-amazon.com/images/I/51IqUfxT...   \n",
       "2419  http://ecx.images-amazon.com/images/I/51nM60Wv...   \n",
       "\n",
       "                                                related salesRank  \\\n",
       "0     {'also_bought': ['B000MFJNVK', 'B000Y52D5G', '...        {}   \n",
       "1     {'also_bought': ['B003155XYO', 'B004F9QBGE', '...        {}   \n",
       "2     {'also_bought': [], 'bought_together': [], 'bu...        {}   \n",
       "3     {'also_bought': ['B00000JZKB', 'B001PLII3E', '...       NaN   \n",
       "4     {'also_bought': ['B00004Z5QO', 'B0000721Z3', '...       NaN   \n",
       "...                                                 ...       ...   \n",
       "2415  {'also_bought': ['B000SDY14K', 'B00I4XMEYA'], ...        {}   \n",
       "2416  {'also_viewed': ['B004F9QBGE', 'B00DGY5IP4', '...       NaN   \n",
       "2417  {'also_viewed': ['B004QWZINW'], 'buy_after_vie...       NaN   \n",
       "2418  {'also_viewed': ['B000Z8W7I0', 'B002ONB4GA', '...       NaN   \n",
       "2419  {'also_viewed': ['B003MZORR2', 'B00I9D5IFM', '...       NaN   \n",
       "\n",
       "                                             categories  \\\n",
       "0     [[Office Products, Office Electronics, Calcula...   \n",
       "1     [[Office Products, Office Electronics, Calcula...   \n",
       "2     [[Office Products, Office Electronics, Calcula...   \n",
       "3     [[Office Products, Office Electronics, Calcula...   \n",
       "4     [[Office Products, Office & School Supplies, P...   \n",
       "...                                                 ...   \n",
       "2415  [[Office Products, Office Electronics, Printer...   \n",
       "2416  [[Office Products, Office & School Supplies, P...   \n",
       "2417  [[Office Products, Office & School Supplies, F...   \n",
       "2418  [[Office Products, Office & School Supplies, F...   \n",
       "2419  [[Office Products, Office & School Supplies, E...   \n",
       "\n",
       "                                                  title              brand  \\\n",
       "0     Texas Instruments TI-1795 SV Standard Function...  Texas Instruments   \n",
       "1     Texas Instruments TI-30X IIS 2-Line Scientific...  Texas Instruments   \n",
       "2           Texas Instruments TI-83 Graphing Calculator  Texas Instruments   \n",
       "3                           HP 12C Financial Calculator                 HP   \n",
       "4     Avery Half-Fold Greeting Cards for Inkjet Prin...              Avery   \n",
       "...                                                 ...                ...   \n",
       "2415  Brother Printer MFC-L8850CDW Wireless Color La...            Brother   \n",
       "2416  Classic Glow Filler Paper, Wide Ruled,600, MAD...                NaN   \n",
       "2417  SentrySafe CHW20221 Medium Chest Safe, Charcoa...                NaN   \n",
       "2418  SentrySafe FHW40220 Large File Safe, Charcoal ...                NaN   \n",
       "2419  Accuteck ShipPro 110lbs x 0.1 oz. Digital Ship...   Accuteck ShipPro   \n",
       "\n",
       "      l2_category  l1_brand  \n",
       "0               4         1  \n",
       "1               4         1  \n",
       "2               4         1  \n",
       "3               4         2  \n",
       "4              25         3  \n",
       "...           ...       ...  \n",
       "2415           29         6  \n",
       "2416           25         0  \n",
       "2417           19         0  \n",
       "2418           19         0  \n",
       "2419           16       341  \n",
       "\n",
       "[2420 rows x 11 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "useful_meta_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>i_category</th>\n",
       "      <th>i_brand</th>\n",
       "      <th>r_complement</th>\n",
       "      <th>r_substitute</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>170</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>[1532, 1536, 2156, 1344, 411, 388, 161, 21, 73...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>[1295, 1856, 2078, 561, 805, 1759, 21, 635, 65...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>[14, 1112, 478, 114, 434, 50]</td>\n",
       "      <td>[14, 180, 114, 25, 50]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>859</td>\n",
       "      <td>25</td>\n",
       "      <td>3</td>\n",
       "      <td>[157, 12, 1073, 1177, 341, 1142, 293, 1722, 64...</td>\n",
       "      <td>[846, 1854, 1722, 1235, 647, 1101, 1073, 558]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2415</th>\n",
       "      <td>2399</td>\n",
       "      <td>29</td>\n",
       "      <td>6</td>\n",
       "      <td>[392, 2358]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2416</th>\n",
       "      <td>2418</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[1856, 2159, 1292, 566, 1016, 1705, 559, 1759,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2417</th>\n",
       "      <td>2401</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[1698]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2418</th>\n",
       "      <td>2406</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[1260, 681, 2401, 1698]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2419</th>\n",
       "      <td>2417</td>\n",
       "      <td>16</td>\n",
       "      <td>341</td>\n",
       "      <td>[]</td>\n",
       "      <td>[1952, 2310, 1399, 2204, 614, 915, 1739, 1869,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2420 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id  i_category  i_brand  \\\n",
       "0         170           4        1   \n",
       "1          25           4        1   \n",
       "2          31           4        1   \n",
       "3           6           4        2   \n",
       "4         859          25        3   \n",
       "...       ...         ...      ...   \n",
       "2415     2399          29        6   \n",
       "2416     2418          25        0   \n",
       "2417     2401          19        0   \n",
       "2418     2406          19        0   \n",
       "2419     2417          16      341   \n",
       "\n",
       "                                           r_complement  \\\n",
       "0     [1532, 1536, 2156, 1344, 411, 388, 161, 21, 73...   \n",
       "1     [1295, 1856, 2078, 561, 805, 1759, 21, 635, 65...   \n",
       "2                                                    []   \n",
       "3                         [14, 1112, 478, 114, 434, 50]   \n",
       "4     [157, 12, 1073, 1177, 341, 1142, 293, 1722, 64...   \n",
       "...                                                 ...   \n",
       "2415                                        [392, 2358]   \n",
       "2416                                                 []   \n",
       "2417                                                 []   \n",
       "2418                                                 []   \n",
       "2419                                                 []   \n",
       "\n",
       "                                           r_substitute  \n",
       "0                                                    []  \n",
       "1                                                    []  \n",
       "2                                                    []  \n",
       "3                                [14, 180, 114, 25, 50]  \n",
       "4         [846, 1854, 1722, 1235, 647, 1101, 1073, 558]  \n",
       "...                                                 ...  \n",
       "2415                                                 []  \n",
       "2416  [1856, 2159, 1292, 566, 1016, 1705, 559, 1759,...  \n",
       "2417                                             [1698]  \n",
       "2418                            [1260, 681, 2401, 1698]  \n",
       "2419  [1952, 2310, 1399, 2204, 614, 915, 1739, 1869,...  \n",
       "\n",
       "[2420 rows x 5 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_meta_data = dict()\n",
    "for idx in range(len(useful_meta_df)):\n",
    "    info = useful_meta_df.iloc[idx]['related']\n",
    "    item_meta_data[idx] = {\n",
    "        'item_id': item2id[useful_meta_df.iloc[idx]['asin']],\n",
    "        'i_category': useful_meta_df.iloc[idx]['l2_category'],\n",
    "        'i_brand': useful_meta_df.iloc[idx]['l1_brand'],\n",
    "        'r_complement': list(map(lambda x: item2id[x], info['also_bought'])) if 'also_bought' in info else [],\n",
    "        'r_substitute': list(map(lambda x: item2id[x], info['also_viewed'])) if 'also_viewed' in info else [],\n",
    "    }\n",
    "\n",
    "item_meta_df = pd.DataFrame.from_dict(item_meta_data, orient='index')\n",
    "item_meta_df = item_meta_df[['item_id', 'i_category', 'i_brand', 'r_complement', 'r_substitute']]\n",
    "item_meta_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save results\n",
    "\n",
    "item_meta_df.to_csv(os.path.join(RAW_PATH, 'item_meta.csv'), sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>imUrl</th>\n",
       "      <th>related</th>\n",
       "      <th>salesRank</th>\n",
       "      <th>categories</th>\n",
       "      <th>title</th>\n",
       "      <th>brand</th>\n",
       "      <th>l2_category</th>\n",
       "      <th>l1_brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B00000JBLU</td>\n",
       "      <td>The classic mini-desktop calculator for home, ...</td>\n",
       "      <td>10.05</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51cpdDRB...</td>\n",
       "      <td>{'also_bought': ['B000MFJNVK', 'B000Y52D5G', '...</td>\n",
       "      <td>{}</td>\n",
       "      <td>[[Office Products, Office Electronics, Calcula...</td>\n",
       "      <td>Texas Instruments TI-1795 SV Standard Function...</td>\n",
       "      <td>Texas Instruments</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B00000JBNX</td>\n",
       "      <td>The new two-line TI-30X IIS with the memory re...</td>\n",
       "      <td>8.38</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/411XXAyD...</td>\n",
       "      <td>{'also_bought': ['B003155XYO', 'B004F9QBGE', '...</td>\n",
       "      <td>{}</td>\n",
       "      <td>[[Office Products, Office Electronics, Calcula...</td>\n",
       "      <td>Texas Instruments TI-30X IIS 2-Line Scientific...</td>\n",
       "      <td>Texas Instruments</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B00000JBO8</td>\n",
       "      <td>Perhaps if we'd had this calculator in high sc...</td>\n",
       "      <td>104.99</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51QZKWB0...</td>\n",
       "      <td>{'also_bought': [], 'bought_together': [], 'bu...</td>\n",
       "      <td>{}</td>\n",
       "      <td>[[Office Products, Office Electronics, Calcula...</td>\n",
       "      <td>Texas Instruments TI-83 Graphing Calculator</td>\n",
       "      <td>Texas Instruments</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B00000JBLH</td>\n",
       "      <td>If you bought yourself a financial calculator ...</td>\n",
       "      <td>51.94</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/415w5D26...</td>\n",
       "      <td>{'also_bought': ['B00000JZKB', 'B001PLII3E', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office Electronics, Calcula...</td>\n",
       "      <td>HP 12C Financial Calculator</td>\n",
       "      <td>HP</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B00000JFNV</td>\n",
       "      <td>Perfect for invitations, thank you notes, movi...</td>\n",
       "      <td>8.42</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51q11PWY...</td>\n",
       "      <td>{'also_bought': ['B00004Z5QO', 'B0000721Z3', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, P...</td>\n",
       "      <td>Avery Half-Fold Greeting Cards for Inkjet Prin...</td>\n",
       "      <td>Avery</td>\n",
       "      <td>25</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2415</th>\n",
       "      <td>B00K5UZP2S</td>\n",
       "      <td></td>\n",
       "      <td>529.99</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41KZwQuZ...</td>\n",
       "      <td>{'also_bought': ['B000SDY14K', 'B00I4XMEYA'], ...</td>\n",
       "      <td>{}</td>\n",
       "      <td>[[Office Products, Office Electronics, Printer...</td>\n",
       "      <td>Brother Printer MFC-L8850CDW Wireless Color La...</td>\n",
       "      <td>Brother</td>\n",
       "      <td>29</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2416</th>\n",
       "      <td>B00KLLBEPI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.99</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41a7pEjw...</td>\n",
       "      <td>{'also_viewed': ['B004F9QBGE', 'B00DGY5IP4', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, P...</td>\n",
       "      <td>Classic Glow Filler Paper, Wide Ruled,600, MAD...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2417</th>\n",
       "      <td>B00KOO594O</td>\n",
       "      <td></td>\n",
       "      <td>54.42</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/41wjtN1j...</td>\n",
       "      <td>{'also_viewed': ['B004QWZINW'], 'buy_after_vie...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, F...</td>\n",
       "      <td>SentrySafe CHW20221 Medium Chest Safe, Charcoa...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2418</th>\n",
       "      <td>B00KOO599O</td>\n",
       "      <td></td>\n",
       "      <td>79.43</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51IqUfxT...</td>\n",
       "      <td>{'also_viewed': ['B000Z8W7I0', 'B002ONB4GA', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, F...</td>\n",
       "      <td>SentrySafe FHW40220 Large File Safe, Charcoal ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2419</th>\n",
       "      <td>B00KYA0RC2</td>\n",
       "      <td>Accuteck ShipPro W-8580 110lbs x 0.1 oz. Black...</td>\n",
       "      <td>25.99</td>\n",
       "      <td>http://ecx.images-amazon.com/images/I/51nM60Wv...</td>\n",
       "      <td>{'also_viewed': ['B003MZORR2', 'B00I9D5IFM', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[Office Products, Office &amp; School Supplies, E...</td>\n",
       "      <td>Accuteck ShipPro 110lbs x 0.1 oz. Digital Ship...</td>\n",
       "      <td>Accuteck ShipPro</td>\n",
       "      <td>16</td>\n",
       "      <td>341</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2420 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            asin                                        description   price  \\\n",
       "0     B00000JBLU  The classic mini-desktop calculator for home, ...   10.05   \n",
       "1     B00000JBNX  The new two-line TI-30X IIS with the memory re...    8.38   \n",
       "2     B00000JBO8  Perhaps if we'd had this calculator in high sc...  104.99   \n",
       "3     B00000JBLH  If you bought yourself a financial calculator ...   51.94   \n",
       "4     B00000JFNV  Perfect for invitations, thank you notes, movi...    8.42   \n",
       "...          ...                                                ...     ...   \n",
       "2415  B00K5UZP2S                                                     529.99   \n",
       "2416  B00KLLBEPI                                                NaN   10.99   \n",
       "2417  B00KOO594O                                                      54.42   \n",
       "2418  B00KOO599O                                                      79.43   \n",
       "2419  B00KYA0RC2  Accuteck ShipPro W-8580 110lbs x 0.1 oz. Black...   25.99   \n",
       "\n",
       "                                                  imUrl  \\\n",
       "0     http://ecx.images-amazon.com/images/I/51cpdDRB...   \n",
       "1     http://ecx.images-amazon.com/images/I/411XXAyD...   \n",
       "2     http://ecx.images-amazon.com/images/I/51QZKWB0...   \n",
       "3     http://ecx.images-amazon.com/images/I/415w5D26...   \n",
       "4     http://ecx.images-amazon.com/images/I/51q11PWY...   \n",
       "...                                                 ...   \n",
       "2415  http://ecx.images-amazon.com/images/I/41KZwQuZ...   \n",
       "2416  http://ecx.images-amazon.com/images/I/41a7pEjw...   \n",
       "2417  http://ecx.images-amazon.com/images/I/41wjtN1j...   \n",
       "2418  http://ecx.images-amazon.com/images/I/51IqUfxT...   \n",
       "2419  http://ecx.images-amazon.com/images/I/51nM60Wv...   \n",
       "\n",
       "                                                related salesRank  \\\n",
       "0     {'also_bought': ['B000MFJNVK', 'B000Y52D5G', '...        {}   \n",
       "1     {'also_bought': ['B003155XYO', 'B004F9QBGE', '...        {}   \n",
       "2     {'also_bought': [], 'bought_together': [], 'bu...        {}   \n",
       "3     {'also_bought': ['B00000JZKB', 'B001PLII3E', '...       NaN   \n",
       "4     {'also_bought': ['B00004Z5QO', 'B0000721Z3', '...       NaN   \n",
       "...                                                 ...       ...   \n",
       "2415  {'also_bought': ['B000SDY14K', 'B00I4XMEYA'], ...        {}   \n",
       "2416  {'also_viewed': ['B004F9QBGE', 'B00DGY5IP4', '...       NaN   \n",
       "2417  {'also_viewed': ['B004QWZINW'], 'buy_after_vie...       NaN   \n",
       "2418  {'also_viewed': ['B000Z8W7I0', 'B002ONB4GA', '...       NaN   \n",
       "2419  {'also_viewed': ['B003MZORR2', 'B00I9D5IFM', '...       NaN   \n",
       "\n",
       "                                             categories  \\\n",
       "0     [[Office Products, Office Electronics, Calcula...   \n",
       "1     [[Office Products, Office Electronics, Calcula...   \n",
       "2     [[Office Products, Office Electronics, Calcula...   \n",
       "3     [[Office Products, Office Electronics, Calcula...   \n",
       "4     [[Office Products, Office & School Supplies, P...   \n",
       "...                                                 ...   \n",
       "2415  [[Office Products, Office Electronics, Printer...   \n",
       "2416  [[Office Products, Office & School Supplies, P...   \n",
       "2417  [[Office Products, Office & School Supplies, F...   \n",
       "2418  [[Office Products, Office & School Supplies, F...   \n",
       "2419  [[Office Products, Office & School Supplies, E...   \n",
       "\n",
       "                                                  title              brand  \\\n",
       "0     Texas Instruments TI-1795 SV Standard Function...  Texas Instruments   \n",
       "1     Texas Instruments TI-30X IIS 2-Line Scientific...  Texas Instruments   \n",
       "2           Texas Instruments TI-83 Graphing Calculator  Texas Instruments   \n",
       "3                           HP 12C Financial Calculator                 HP   \n",
       "4     Avery Half-Fold Greeting Cards for Inkjet Prin...              Avery   \n",
       "...                                                 ...                ...   \n",
       "2415  Brother Printer MFC-L8850CDW Wireless Color La...            Brother   \n",
       "2416  Classic Glow Filler Paper, Wide Ruled,600, MAD...                NaN   \n",
       "2417  SentrySafe CHW20221 Medium Chest Safe, Charcoa...                NaN   \n",
       "2418  SentrySafe FHW40220 Large File Safe, Charcoal ...                NaN   \n",
       "2419  Accuteck ShipPro 110lbs x 0.1 oz. Digital Ship...   Accuteck ShipPro   \n",
       "\n",
       "      l2_category  l1_brand  \n",
       "0               4         1  \n",
       "1               4         1  \n",
       "2               4         1  \n",
       "3               4         2  \n",
       "4              25         3  \n",
       "...           ...       ...  \n",
       "2415           29         6  \n",
       "2416           25         0  \n",
       "2417           19         0  \n",
       "2418           19         0  \n",
       "2419           16       341  \n",
       "\n",
       "[2420 rows x 11 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "useful_meta_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2420/2420 [00:00<00:00, 9308.73it/s]\n"
     ]
    }
   ],
   "source": [
    "import html\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "def clean_text(text):\n",
    "    text = html.unescape(text)\n",
    "    text = re.sub(r'[\"\\n\\r]*', '', text)\n",
    "    return text\n",
    "outf = open(os.path.join(RAW_PATH, f'{DATASET}.text'),\"w\")\n",
    "outf.write(f\"id\\ttext\\n\")\n",
    "for index, row in tqdm(useful_meta_df.iterrows(),total=len(useful_meta_df)):\n",
    "    item_id, title, categories, brand = row['asin'], row[\"title\"], row['categories'],row['brand']\n",
    "    text = \"\"\n",
    "    if not pd.isnull(title):\n",
    "        title = \" \".join(title.strip().split()[:32])\n",
    "        title = clean_text(title)\n",
    "        text += title\n",
    "    if len(categories) > 0:\n",
    "        for cate in categories:\n",
    "            if cate[0] == \"Electronics\":\n",
    "                category = clean_text(cate[-1])\n",
    "                text += \" \" + category\n",
    "    #     item2category[item_id] = category\n",
    "    if not pd.isnull(brand):\n",
    "        brand = clean_text(brand)\n",
    "        text += \" \" + brand\n",
    "        # item2brand[item_id] = clean_text(brand)\n",
    "    outf.write(f\"{item_id}\\t{text}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_unit2index(file):\n",
    "    unit2index = dict()\n",
    "    with open(file, 'r') as fp:\n",
    "        for line in fp:\n",
    "            unit, index = line.strip().split('\\t')\n",
    "            unit2index[unit] = int(index)\n",
    "    return unit2index\n",
    "\n",
    "\n",
    "def write_remap_index(unit2index, file):\n",
    "    with open(file, 'w') as fp:\n",
    "        for unit in unit2index:\n",
    "            fp.write(unit + '\\t' + str(unit2index[unit]) + '\\n')\n",
    "\n",
    "write_remap_index(user2id, os.path.join(RAW_PATH, f'{DATASET}.user2index'))\n",
    "write_remap_index(item2id, os.path.join(RAW_PATH, f'{DATASET}.item2index'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "prompt",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
